# coding=utf-8
from bs4 import BeautifulSoup
import re
import urlparse
import html_downloader
import urllib2
import MySQLdb


class HtmlParser(object):
    def __init__(self):
        self.downloader = html_downloader.HtmlDownloader()

    def _get_new_urls(self, soup):  # 在soup里获取页面新的urls
        new_urls = set()  # 获取到新的url列表的集合
        pattern = re.compile(ur'[\u653b]+[\u51fb]+[\u4e00-\u9fa5]+[\u7a0b]')
        links = soup.find_all('div', class_='news_inner news-list')
        conn = MySQLdb.connect(host='xuliugen.vicp.io', user='ljn',
                               passwd='123456', port=33006, db='attack-process', charset="utf8")
        cursor = conn.cursor()
        sql = "insert into test(title,process,link) values(%s,%s,%s)"
        for link in links:
            title = link.find('img')['title']
            new_url = link.find('a')['href']
            print 'craw:%s' % (new_url)
            html_cont1 = self.downloader.download(new_url)
            soup1 = BeautifulSoup(html_cont1, 'lxml', from_encoding='utf-8')
            # test_url = 'http://www.freebuf.com/vuls/148575.html'
            # print 'craw:%s' % (test_url)
            # test_cont1 = self.downloader.download(test_url)
            # soup1 = BeautifulSoup(test_cont1, 'lxml', from_encoding='utf-8')
            content = soup1.find('div', id='contenttxt')
            if content != None:
                content1 = content.find_all(True)
                count = 0
                for s in content1:
                    count += 1
                    if pattern.search(s.text) and (s.name == ('h2' or 'h1' or 'h3' or 'h4' or 'h5')):  # 正则表达式需要修改
                        # if s.text.encode('utf8')==str.encode('utf8'):
                        str = s.name  # 存储匹配到的过程名字的字体，如h1 h2 h3
                        process = ""
                        for i in range(count, len(content1)):
                            if (content1[i].name == str):  # 当再次遇到相同标题字体时，结束
                                break
                            if (content1[i].text != "" and content1[i].name == 'p'):
                                process = process + content1[i].text + "\n"
                        try:
                            cursor.execute(sql, (title, process,new_url))
                            conn.commit()
                            print("成功")
                        except:
                            print("错误")
                        break
                        # print 'the %d step: %s' % (i - count + 1, content1[i].text)
            else:
                continue
        link_next = soup.find('div', class_='news-more')
        if (link_next):
            link_next = link_next.find('a')['href']
            return link_next
        else:
            return ""
            #     cont_next = self.downloader.download(link_next)
            #     soup_next = BeautifulSoup(cont_next,'lxml',from_encoding='utf-8')
            #     self._get_new_urls(soup_next)


            #  爬取https://nvd.nist.gov/网站
            # page_url = "https://nvd.nist.gov/"
            # links = soup.find_all('ul', class_='list-inline')
            # for link in links:
            #     new_url = link.find('a')['href']
            #     new_full_url = urlparse.urljoin(page_url, new_url)
            #     print 'craw : %s' % (new_full_url)
            #     html_cont1 = self.downloader.download(new_full_url)
            #     soup1 = BeautifulSoup(html_cont1, 'html.parser', from_encoding='utf-8')
            #     page_url1 = "https://nvd.nist.gov/"
            #     links1 = soup1.find_all('span', class_='col-md-2')
            #     for link1 in links1:
            #         new_url1 = link1.find('a')['href']
            #         new_full_url1 = urlparse.urljoin(page_url, new_url1)
            #         html_cont2 = self.downloader.download(new_full_url1)
            #         soup2 = BeautifulSoup(html_cont2, 'html.parser', from_encoding='utf-8')
            #         data = {}
            #         id = soup2.find('title').text
            #         id1 = id.replace('\t', '').replace('\n', '').replace(' ', '')
            #         release_time = soup2.find('dd',attrs={'data-testid':'vuln-published-on'}).text
            #         modified_time = soup2.find('dd',attrs={'data-testid':'vuln-last-modified-on'}).text
            #         source = soup2.find('dd',attrs={'data-testid':'vuln-source'}).text
            #         description = soup2.find('p', attrs={'data-testid':'vuln-description'}).text
            #         data['id'] = id1
            #         data['release_time'] = release_time
            #         data['modified_time'] = modified_time
            #         data['source'] = source
            #         data['description'] = description
            #         print 'id1:%s,release_time :%s,modified_time:%s,source:%s,description:%s' % (id1,release_time,modified_time,source,description)

    # def _get_new_urlss(self, soup):
    #     new_urls = set()
    #
    #     page_url = "https://nvd.nist.gov/"
    #     links = soup.find_all('span', class_='col-md-2')
    #     for link in links:
    #         new_url = link.find('a')['href']
    #         new_full_url = urlparse.urljoin(page_url, new_url)
    #         new_urls.add(new_full_url)
    #     return new_urls

    # def _get_data(self, soup):
    #     data = {}
    #     id = soup.find('title')
    #     print id

    def parse(self, html_cont):
        soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
        new_urls = self._get_new_urls(soup)
        # count = 1
        # while new_urls != None:
        #     html_cont1 = self.downloader.download(new_urls)
        #     soup = BeautifulSoup(html_cont1, 'lxml', from_encoding='utf-8')
        #     new_urls = self._get_new_urls(soup)
        return new_urls
